{
  "cells": [
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# `waterfall_chart`\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "tags": [
          "hide_input"
        ]
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "<style>\n",
              "  #altair-viz-ad2e9447155e4aefaf727dfef09cf865.vega-embed {\n",
              "    width: 100%;\n",
              "    display: flex;\n",
              "  }\n",
              "\n",
              "  #altair-viz-ad2e9447155e4aefaf727dfef09cf865.vega-embed details,\n",
              "  #altair-viz-ad2e9447155e4aefaf727dfef09cf865.vega-embed details summary {\n",
              "    position: relative;\n",
              "  }\n",
              "</style>\n",
              "<div id=\"altair-viz-ad2e9447155e4aefaf727dfef09cf865\"></div>\n",
              "<script type=\"text/javascript\">\n",
              "  var VEGA_DEBUG = (typeof VEGA_DEBUG == \"undefined\") ? {} : VEGA_DEBUG;\n",
              "  (function(spec, embedOpt){\n",
              "    let outputDiv = document.currentScript.previousElementSibling;\n",
              "    if (outputDiv.id !== \"altair-viz-ad2e9447155e4aefaf727dfef09cf865\") {\n",
              "      outputDiv = document.getElementById(\"altair-viz-ad2e9447155e4aefaf727dfef09cf865\");\n",
              "    }\n",
              "    const paths = {\n",
              "      \"vega\": \"https://cdn.jsdelivr.net/npm/vega@5?noext\",\n",
              "      \"vega-lib\": \"https://cdn.jsdelivr.net/npm/vega-lib?noext\",\n",
              "      \"vega-lite\": \"https://cdn.jsdelivr.net/npm/vega-lite@5.17.0?noext\",\n",
              "      \"vega-embed\": \"https://cdn.jsdelivr.net/npm/vega-embed@6?noext\",\n",
              "    };\n",
              "\n",
              "    function maybeLoadScript(lib, version) {\n",
              "      var key = `${lib.replace(\"-\", \"\")}_version`;\n",
              "      return (VEGA_DEBUG[key] == version) ?\n",
              "        Promise.resolve(paths[lib]) :\n",
              "        new Promise(function(resolve, reject) {\n",
              "          var s = document.createElement('script');\n",
              "          document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
              "          s.async = true;\n",
              "          s.onload = () => {\n",
              "            VEGA_DEBUG[key] = version;\n",
              "            return resolve(paths[lib]);\n",
              "          };\n",
              "          s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
              "          s.src = paths[lib];\n",
              "        });\n",
              "    }\n",
              "\n",
              "    function showError(err) {\n",
              "      outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
              "      throw err;\n",
              "    }\n",
              "\n",
              "    function displayChart(vegaEmbed) {\n",
              "      vegaEmbed(outputDiv, spec, embedOpt)\n",
              "        .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
              "    }\n",
              "\n",
              "    if(typeof define === \"function\" && define.amd) {\n",
              "      requirejs.config({paths});\n",
              "      require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
              "    } else {\n",
              "      maybeLoadScript(\"vega\", \"5\")\n",
              "        .then(() => maybeLoadScript(\"vega-lite\", \"5.17.0\"))\n",
              "        .then(() => maybeLoadScript(\"vega-embed\", \"6\"))\n",
              "        .catch(showError)\n",
              "        .then(() => displayChart(vegaEmbed));\n",
              "    }\n",
              "  })({\"config\": {\"view\": {\"continuousWidth\": 400, \"continuousHeight\": 300}}, \"layer\": [{\"layer\": [{\"mark\": \"rule\", \"encoding\": {\"color\": {\"value\": \"black\"}, \"size\": {\"value\": 0.5}, \"y\": {\"field\": \"zero\", \"type\": \"quantitative\"}}}, {\"mark\": {\"type\": \"bar\", \"width\": 60}, \"encoding\": {\"color\": {\"condition\": {\"test\": \"(datum.log2_bayes_factor < 0)\", \"value\": \"red\"}, \"value\": \"green\"}, \"opacity\": {\"condition\": {\"test\": \"datum.column_name == 'Prior match weight' || datum.column_name == 'Final score'\", \"value\": 1}, \"value\": 0.5}, \"tooltip\": [{\"field\": \"column_name\", \"title\": \"Comparison column\", \"type\": \"nominal\"}, {\"field\": \"value_l\", \"title\": \"Value (L)\", \"type\": \"nominal\"}, {\"field\": \"value_r\", \"title\": \"Value (R)\", \"type\": \"nominal\"}, {\"field\": \"label_for_charts\", \"title\": \"Label\", \"type\": \"ordinal\"}, {\"field\": \"sql_condition\", \"title\": \"SQL condition\", \"type\": \"nominal\"}, {\"field\": \"comparison_vector_value\", \"title\": \"Comparison vector value\", \"type\": \"nominal\"}, {\"field\": \"bayes_factor\", \"format\": \",.4f\", \"title\": \"Bayes factor = m/u\", \"type\": \"quantitative\"}, {\"field\": \"log2_bayes_factor\", \"format\": \",.4f\", \"title\": \"Match weight = log2(m/u)\", \"type\": \"quantitative\"}, {\"field\": \"prob\", \"format\": \".4f\", \"title\": \"Cumulative match probability\", \"type\": \"quantitative\"}, {\"field\": \"bayes_factor_description\", \"title\": \"Match weight description\", \"type\": \"nominal\"}], \"x\": {\"axis\": {\"grid\": true, \"labelAlign\": \"center\", \"labelAngle\": -20, \"labelExpr\": \"datum.value == 'Prior' || datum.value == 'Final score' ? '' : datum.value\", \"labelPadding\": 10, \"tickBand\": \"extent\", \"title\": \"Column\"}, \"field\": \"column_name\", \"sort\": {\"field\": \"bar_sort_order\", \"order\": \"ascending\"}, \"type\": \"nominal\"}, \"y\": {\"axis\": {\"grid\": false, \"orient\": \"left\", \"title\": \"Match Weight\"}, \"field\": \"previous_sum\", \"type\": \"quantitative\"}, \"y2\": {\"field\": \"sum\"}}}, {\"mark\": {\"type\": \"text\", \"fontWeight\": \"bold\"}, \"encoding\": {\"color\": {\"value\": \"white\"}, \"text\": {\"condition\": {\"test\": \"abs(datum.log2_bayes_factor) > 1\", \"field\": \"log2_bayes_factor\", \"format\": \".2f\", \"type\": \"nominal\"}, \"value\": \"\"}, \"x\": {\"axis\": {\"labelAngle\": -20, \"title\": \"Column\"}, \"field\": \"column_name\", \"sort\": {\"field\": \"bar_sort_order\", \"order\": \"ascending\"}, \"type\": \"nominal\"}, \"y\": {\"axis\": {\"orient\": \"left\"}, \"field\": \"center\", \"type\": \"quantitative\"}}}, {\"mark\": {\"type\": \"text\", \"baseline\": \"bottom\", \"dy\": -25, \"fontWeight\": \"bold\"}, \"encoding\": {\"color\": {\"value\": \"black\"}, \"text\": {\"field\": \"column_name\", \"type\": \"nominal\"}, \"x\": {\"axis\": {\"labelAngle\": -20, \"title\": \"Column\"}, \"field\": \"column_name\", \"sort\": {\"field\": \"bar_sort_order\", \"order\": \"ascending\"}, \"type\": \"nominal\"}, \"y\": {\"field\": \"sum_top\", \"type\": \"quantitative\"}}}, {\"mark\": {\"type\": \"text\", \"baseline\": \"bottom\", \"dy\": -13, \"fontSize\": 8}, \"encoding\": {\"color\": {\"value\": \"grey\"}, \"text\": {\"field\": \"value_l\", \"type\": \"nominal\"}, \"x\": {\"axis\": {\"labelAngle\": -20, \"title\": \"Column\"}, \"field\": \"column_name\", \"sort\": {\"field\": \"bar_sort_order\", \"order\": \"ascending\"}, \"type\": \"nominal\"}, \"y\": {\"field\": \"sum_top\", \"type\": \"quantitative\"}}}, {\"mark\": {\"type\": \"text\", \"baseline\": \"bottom\", \"dy\": -5, \"fontSize\": 8}, \"encoding\": {\"color\": {\"value\": \"grey\"}, \"text\": {\"field\": \"value_r\", \"type\": \"nominal\"}, \"x\": {\"axis\": {\"labelAngle\": -20, \"title\": \"Column\"}, \"field\": \"column_name\", \"sort\": {\"field\": \"bar_sort_order\", \"order\": \"ascending\"}, \"type\": \"nominal\"}, \"y\": {\"field\": \"sum_top\", \"type\": \"quantitative\"}}}]}, {\"mark\": {\"type\": \"rule\", \"color\": \"black\", \"strokeWidth\": 2, \"x2Offset\": 30, \"xOffset\": -30}, \"encoding\": {\"x\": {\"axis\": {\"labelAngle\": -20, \"title\": \"Column\"}, \"field\": \"column_name\", \"sort\": {\"field\": \"bar_sort_order\", \"order\": \"ascending\"}, \"type\": \"nominal\"}, \"x2\": {\"field\": \"lead\"}, \"y\": {\"axis\": {\"labelExpr\": \"format(1 / (1 + pow(2, -1*datum.value)), '.2r')\", \"orient\": \"right\", \"title\": \"Probability\"}, \"field\": \"sum\", \"scale\": {\"zero\": false}, \"type\": \"quantitative\"}}}], \"data\": {\"name\": \"data-61650774e7af395798e3bd6248af495f\"}, \"height\": 450, \"params\": [{\"name\": \"record_number\", \"bind\": {\"input\": \"range\", \"max\": 4, \"min\": 0, \"step\": 1}, \"value\": 0}], \"resolve\": {\"axis\": {\"y\": \"independent\"}}, \"title\": {\"text\": \"Match weights waterfall chart\", \"subtitle\": \"How each comparison contributes to the final match score\"}, \"transform\": [{\"filter\": \"(datum.record_number == record_number)\"}, {\"window\": [{\"op\": \"sum\", \"field\": \"log2_bayes_factor\", \"as\": \"sum\"}, {\"op\": \"lead\", \"field\": \"column_name\", \"as\": \"lead\"}], \"frame\": [null, 0]}, {\"calculate\": \"datum.column_name === \\\"Final score\\\" ? datum.sum - datum.log2_bayes_factor : datum.sum\", \"as\": \"sum\"}, {\"calculate\": \"datum.lead === null ? datum.column_name : datum.lead\", \"as\": \"lead\"}, {\"calculate\": \"datum.column_name === \\\"Final score\\\" || datum.column_name === \\\"Prior match weight\\\" ? 0 : datum.sum - datum.log2_bayes_factor\", \"as\": \"previous_sum\"}, {\"calculate\": \"datum.sum > datum.previous_sum ? datum.column_name : \\\"\\\"\", \"as\": \"top_label\"}, {\"calculate\": \"datum.sum < datum.previous_sum ? datum.column_name : \\\"\\\"\", \"as\": \"bottom_label\"}, {\"calculate\": \"datum.sum > datum.previous_sum ? datum.sum : datum.previous_sum\", \"as\": \"sum_top\"}, {\"calculate\": \"datum.sum < datum.previous_sum ? datum.sum : datum.previous_sum\", \"as\": \"sum_bottom\"}, {\"calculate\": \"(datum.sum + datum.previous_sum) / 2\", \"as\": \"center\"}, {\"calculate\": \"(datum.log2_bayes_factor > 0 ? \\\"+\\\" : \\\"\\\") + datum.log2_bayes_factor\", \"as\": \"text_log2_bayes_factor\"}, {\"calculate\": \"datum.sum < datum.previous_sum ? 4 : -4\", \"as\": \"dy\"}, {\"calculate\": \"datum.sum < datum.previous_sum ? \\\"top\\\" : \\\"bottom\\\"\", \"as\": \"baseline\"}, {\"calculate\": \"1. / (1 + pow(2, -1.*datum.sum))\", \"as\": \"prob\"}, {\"calculate\": \"0*datum.sum\", \"as\": \"zero\"}], \"width\": {\"step\": 75}, \"$schema\": \"https://vega.github.io/schema/vega-lite/v5.9.3.json\", \"datasets\": {\"data-61650774e7af395798e3bd6248af495f\": [{\"column_name\": \"Prior\", \"label_for_charts\": \"Starting match weight (prior)\", \"sql_condition\": null, \"log2_bayes_factor\": -13.287568102831404, \"bayes_factor\": 0.00010001000100010001, \"comparison_vector_value\": null, \"m_probability\": null, \"u_probability\": null, \"bayes_factor_description\": null, \"value_l\": \"\", \"value_r\": \"\", \"term_frequency_adjustment\": null, \"bar_sort_order\": 0, \"record_number\": 0}, {\"sql_condition\": \"\\\"first_name_l\\\" IS NULL OR \\\"first_name_r\\\" IS NULL\", \"label_for_charts\": \"first_name is NULL\", \"bayes_factor\": 1.0, \"log2_bayes_factor\": 0.0, \"comparison_vector_value\": -1, \"bayes_factor_description\": \"If comparison level is `first_name is null` then comparison is 1.00 times more likely to be a match\", \"column_name\": \"first_name\", \"value_l\": \"None\", \"value_r\": \"Lola\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 1, \"record_number\": 0}, {\"sql_condition\": \"\\\"first_name_l\\\" IS NULL OR \\\"first_name_r\\\" IS NULL\", \"label_for_charts\": \"first_name is NULL\", \"bayes_factor\": 1.0, \"log2_bayes_factor\": 0.0, \"comparison_vector_value\": -1, \"bayes_factor_description\": \"If comparison level is `first_name is null` then comparison is 1.00 times more likely to be a match\", \"column_name\": \"tf_first_name\", \"value_l\": \"\", \"value_r\": \"\", \"term_frequency_adjustment\": true, \"bar_sort_order\": 2, \"record_number\": 0}, {\"sql_condition\": \"\\\"surname_l\\\" = \\\"surname_r\\\"\", \"label_for_charts\": \"Exact match on surname\", \"m_probability\": 0.43118558905152193, \"u_probability\": 0.004889975550122249, \"bayes_factor\": 88.17745296103624, \"log2_bayes_factor\": 6.462337899652856, \"comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `exact match on surname` then comparison is 88.18 times more likely to be a match\", \"column_name\": \"surname\", \"value_l\": \"Taylor\", \"value_r\": \"Taylor\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 3, \"record_number\": 0}, {\"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.26762728052570295, \"u_probability\": 0.9611471471471471, \"bayes_factor\": 0.27844568994463287, \"log2_bayes_factor\": -1.8445321335278149, \"comparison_vector_value\": 0, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is  3.59 times less likely to be a match\", \"column_name\": \"dob\", \"value_l\": \"2016-11-20\", \"value_r\": \"2017-11-21\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 4, \"record_number\": 0}, {\"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.4390042158654446, \"u_probability\": 0.9448524288198547, \"bayes_factor\": 0.46462728197012976, \"log2_bayes_factor\": -1.1058542261314457, \"comparison_vector_value\": 0, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is  2.15 times less likely to be a match\", \"column_name\": \"city\", \"value_l\": \"Abereden\", \"value_r\": \"Aberdeen\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 5, \"record_number\": 0}, {\"sql_condition\": \"jaro_winkler_similarity(\\\"email_l\\\", \\\"email_r\\\") >= 0.88\", \"label_for_charts\": \"Jaro-Winkler distance of email >= 0.88\", \"m_probability\": 0.21355998957004996, \"u_probability\": 0.0009135769109519858, \"bayes_factor\": 233.76246379465897, \"log2_bayes_factor\": 7.868899478733789, \"comparison_vector_value\": 1, \"bayes_factor_description\": \"If comparison level is `jaro-winkler distance of email >= 0.88` then comparison is 233.76 times more likely to be a match\", \"column_name\": \"email\", \"value_l\": \"glolat86@bishop-giles.com\", \"value_r\": \"lolat86@bishop-giles.om\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 6, \"record_number\": 0}, {\"column_name\": \"Final score\", \"label_for_charts\": \"Final score\", \"sql_condition\": null, \"log2_bayes_factor\": -1.9067170841040197, \"bayes_factor\": 0.2666987403313988, \"comparison_vector_value\": null, \"m_probability\": null, \"u_probability\": null, \"bayes_factor_description\": null, \"value_l\": \"\", \"value_r\": \"\", \"term_frequency_adjustment\": null, \"bar_sort_order\": 7, \"record_number\": 0}, {\"column_name\": \"Prior\", \"label_for_charts\": \"Starting match weight (prior)\", \"sql_condition\": null, \"log2_bayes_factor\": -13.287568102831404, \"bayes_factor\": 0.00010001000100010001, \"comparison_vector_value\": null, \"m_probability\": null, \"u_probability\": null, \"bayes_factor_description\": null, \"value_l\": \"\", \"value_r\": \"\", \"term_frequency_adjustment\": null, \"bar_sort_order\": 0, \"record_number\": 1}, {\"sql_condition\": \"\\\"first_name_l\\\" = \\\"first_name_r\\\"\", \"label_for_charts\": \"Exact match on first_name\", \"m_probability\": 0.4874498174690064, \"u_probability\": 0.0057935713975033705, \"bayes_factor\": 84.13632697770215, \"log2_bayes_factor\": 6.394656932643231, \"comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `exact match on first_name` then comparison is 84.14 times more likely to be a match\", \"column_name\": \"first_name\", \"value_l\": \"Eleanor\", \"value_r\": \"Eleanor\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 1, \"record_number\": 1}, {\"sql_condition\": \"\\\"first_name_l\\\" = \\\"first_name_r\\\"\", \"label_for_charts\": \"Term freq adjustment on first_name with weight {cl.tf_adjustment_weight}\", \"m_probability\": null, \"u_probability\": null, \"bayes_factor\": 1.2036144578313253, \"log2_bayes_factor\": 0.2673733415581312, \"comparison_vector_value\": 3, \"bayes_factor_description\": \"Term frequency adjustment on first_name makes comparison 1.20 times more likely to be a match\", \"column_name\": \"tf_first_name\", \"value_l\": \"Eleanor\", \"value_r\": \"Eleanor\", \"term_frequency_adjustment\": true, \"bar_sort_order\": 2, \"record_number\": 1}, {\"sql_condition\": \"\\\"surname_l\\\" IS NULL OR \\\"surname_r\\\" IS NULL\", \"label_for_charts\": \"surname is NULL\", \"bayes_factor\": 1.0, \"log2_bayes_factor\": 0.0, \"comparison_vector_value\": -1, \"bayes_factor_description\": \"If comparison level is `surname is null` then comparison is 1.00 times more likely to be a match\", \"column_name\": \"surname\", \"value_l\": \"None\", \"value_r\": \"Richards\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 3, \"record_number\": 1}, {\"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.26762728052570295, \"u_probability\": 0.9611471471471471, \"bayes_factor\": 0.27844568994463287, \"log2_bayes_factor\": -1.8445321335278149, \"comparison_vector_value\": 0, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is  3.59 times less likely to be a match\", \"column_name\": \"dob\", \"value_l\": \"2024-07-07\", \"value_r\": \"2014-07-10\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 4, \"record_number\": 1}, {\"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.4390042158654446, \"u_probability\": 0.9448524288198547, \"bayes_factor\": 0.46462728197012976, \"log2_bayes_factor\": -1.1058542261314457, \"comparison_vector_value\": 0, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is  2.15 times less likely to be a match\", \"column_name\": \"city\", \"value_l\": \"Manchester\", \"value_r\": \"Mancester\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 5, \"record_number\": 1}, {\"sql_condition\": \"NULLIF(regexp_extract(\\\"email_l\\\", '^[^@]+', 0), '') = NULLIF(regexp_extract(\\\"email_r\\\", '^[^@]+', 0), '')\", \"label_for_charts\": \"Exact match on transformed email\", \"m_probability\": 0.22027963026744735, \"u_probability\": 0.0010390328952024346, \"bayes_factor\": 212.00448155640953, \"log2_bayes_factor\": 7.727950951972963, \"comparison_vector_value\": 2, \"bayes_factor_description\": \"If comparison level is `exact match on transformed email` then comparison is 212.00 times more likely to be a match\", \"column_name\": \"email\", \"value_l\": \"e.richards16@finley.ifo\", \"value_r\": \"e.richards16@finley.info\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 6, \"record_number\": 1}, {\"column_name\": \"Final score\", \"label_for_charts\": \"Final score\", \"sql_condition\": null, \"log2_bayes_factor\": -1.8479732363163406, \"bayes_factor\": 0.2777823353038195, \"comparison_vector_value\": null, \"m_probability\": null, \"u_probability\": null, \"bayes_factor_description\": null, \"value_l\": \"\", \"value_r\": \"\", \"term_frequency_adjustment\": null, \"bar_sort_order\": 7, \"record_number\": 1}, {\"column_name\": \"Prior\", \"label_for_charts\": \"Starting match weight (prior)\", \"sql_condition\": null, \"log2_bayes_factor\": -13.287568102831404, \"bayes_factor\": 0.00010001000100010001, \"comparison_vector_value\": null, \"m_probability\": null, \"u_probability\": null, \"bayes_factor_description\": null, \"value_l\": \"\", \"value_r\": \"\", \"term_frequency_adjustment\": null, \"bar_sort_order\": 0, \"record_number\": 2}, {\"sql_condition\": \"\\\"first_name_l\\\" = \\\"first_name_r\\\"\", \"label_for_charts\": \"Exact match on first_name\", \"m_probability\": 0.4874498174690064, \"u_probability\": 0.0057935713975033705, \"bayes_factor\": 84.13632697770215, \"log2_bayes_factor\": 6.394656932643231, \"comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `exact match on first_name` then comparison is 84.14 times more likely to be a match\", \"column_name\": \"first_name\", \"value_l\": \"Darcy\", \"value_r\": \"Darcy\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 1, \"record_number\": 2}, {\"sql_condition\": \"\\\"first_name_l\\\" = \\\"first_name_r\\\"\", \"label_for_charts\": \"Term freq adjustment on first_name with weight {cl.tf_adjustment_weight}\", \"m_probability\": null, \"u_probability\": null, \"bayes_factor\": 0.8024096385542168, \"log2_bayes_factor\": -0.31758915916302516, \"comparison_vector_value\": 3, \"bayes_factor_description\": \"Term frequency adjustment on first_name makes comparison  1.25 times less likely to be a match\", \"column_name\": \"tf_first_name\", \"value_l\": \"Darcy\", \"value_r\": \"Darcy\", \"term_frequency_adjustment\": true, \"bar_sort_order\": 2, \"record_number\": 2}, {\"sql_condition\": \"\\\"surname_l\\\" IS NULL OR \\\"surname_r\\\" IS NULL\", \"label_for_charts\": \"surname is NULL\", \"bayes_factor\": 1.0, \"log2_bayes_factor\": 0.0, \"comparison_vector_value\": -1, \"bayes_factor_description\": \"If comparison level is `surname is null` then comparison is 1.00 times more likely to be a match\", \"column_name\": \"surname\", \"value_l\": \"None\", \"value_r\": \"Rhodes\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 3, \"record_number\": 2}, {\"sql_condition\": \"damerau_levenshtein(\\\"dob_l\\\", \\\"dob_r\\\") <= 1\", \"label_for_charts\": \"Damerau-Levenshtein distance of dob <= 1\", \"m_probability\": 0.14875392956459, \"u_probability\": 0.0016436436436436436, \"bayes_factor\": 90.50254301767687, \"log2_bayes_factor\": 6.499886425729021, \"comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `damerau-levenshtein distance of dob <= 1` then comparison is 90.50 times more likely to be a match\", \"column_name\": \"dob\", \"value_l\": \"1979-01-16\", \"value_r\": \"1979-01-14\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 4, \"record_number\": 2}, {\"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.4390042158654446, \"u_probability\": 0.9448524288198547, \"bayes_factor\": 0.46462728197012976, \"log2_bayes_factor\": -1.1058542261314457, \"comparison_vector_value\": 0, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is  2.15 times less likely to be a match\", \"column_name\": \"city\", \"value_l\": \"nBirminham\", \"value_r\": \"Birmngham\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 5, \"record_number\": 2}, {\"sql_condition\": \"\\\"email_l\\\" IS NULL OR \\\"email_r\\\" IS NULL\", \"label_for_charts\": \"email is NULL\", \"bayes_factor\": 1.0, \"log2_bayes_factor\": 0.0, \"comparison_vector_value\": -1, \"bayes_factor_description\": \"If comparison level is `email is null` then comparison is 1.00 times more likely to be a match\", \"column_name\": \"email\", \"value_l\": \"drhodes16@johnson-robinson.com\", \"value_r\": \"None\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 6, \"record_number\": 2}, {\"column_name\": \"Final score\", \"label_for_charts\": \"Final score\", \"sql_condition\": null, \"log2_bayes_factor\": -1.816468129753623, \"bayes_factor\": 0.2839151756519555, \"comparison_vector_value\": null, \"m_probability\": null, \"u_probability\": null, \"bayes_factor_description\": null, \"value_l\": \"\", \"value_r\": \"\", \"term_frequency_adjustment\": null, \"bar_sort_order\": 7, \"record_number\": 2}, {\"column_name\": \"Prior\", \"label_for_charts\": \"Starting match weight (prior)\", \"sql_condition\": null, \"log2_bayes_factor\": -13.287568102831404, \"bayes_factor\": 0.00010001000100010001, \"comparison_vector_value\": null, \"m_probability\": null, \"u_probability\": null, \"bayes_factor_description\": null, \"value_l\": \"\", \"value_r\": \"\", \"term_frequency_adjustment\": null, \"bar_sort_order\": 0, \"record_number\": 3}, {\"sql_condition\": \"\\\"first_name_l\\\" = \\\"first_name_r\\\"\", \"label_for_charts\": \"Exact match on first_name\", \"m_probability\": 0.4874498174690064, \"u_probability\": 0.0057935713975033705, \"bayes_factor\": 84.13632697770215, \"log2_bayes_factor\": 6.394656932643231, \"comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `exact match on first_name` then comparison is 84.14 times more likely to be a match\", \"column_name\": \"first_name\", \"value_l\": \"Jackson\", \"value_r\": \"Jackson\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 1, \"record_number\": 3}, {\"sql_condition\": \"\\\"first_name_l\\\" = \\\"first_name_r\\\"\", \"label_for_charts\": \"Term freq adjustment on first_name with weight {cl.tf_adjustment_weight}\", \"m_probability\": null, \"u_probability\": null, \"bayes_factor\": 0.6877796901893287, \"log2_bayes_factor\": -0.539981580499473, \"comparison_vector_value\": 3, \"bayes_factor_description\": \"Term frequency adjustment on first_name makes comparison  1.45 times less likely to be a match\", \"column_name\": \"tf_first_name\", \"value_l\": \"Jackson\", \"value_r\": \"Jackson\", \"term_frequency_adjustment\": true, \"bar_sort_order\": 2, \"record_number\": 3}, {\"sql_condition\": \"jaro_winkler_similarity(\\\"surname_l\\\", \\\"surname_r\\\") >= 0.8\", \"label_for_charts\": \"Jaro-Winkler distance of surname >= 0.8\", \"m_probability\": 0.07560521862782084, \"u_probability\": 0.0039048156407569612, \"bayes_factor\": 19.362045633776585, \"log2_bayes_factor\": 4.275159478791109, \"comparison_vector_value\": 1, \"bayes_factor_description\": \"If comparison level is `jaro-winkler distance of surname >= 0.8` then comparison is 19.36 times more likely to be a match\", \"column_name\": \"surname\", \"value_l\": \"Fisreh\", \"value_r\": \"Fishier\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 3, \"record_number\": 3}, {\"sql_condition\": \"ABS(EPOCH(try_strptime(\\\"dob_l\\\", '%Y-%m-%d')) - EPOCH(try_strptime(\\\"dob_r\\\", '%Y-%m-%d'))) <= 31557600.0\", \"label_for_charts\": \"Abs difference of 'transformed dob <= 1 year'\", \"m_probability\": 0.19388291091910234, \"u_probability\": 0.03546146146146146, \"bayes_factor\": 5.46742584565526, \"log2_bayes_factor\": 2.4508617482261057, \"comparison_vector_value\": 2, \"bayes_factor_description\": \"If comparison level is `abs difference of 'transformed dob <= 1 year'` then comparison is 5.47 times more likely to be a match\", \"column_name\": \"dob\", \"value_l\": \"2004-12-26\", \"value_r\": \"2003-12-28\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 4, \"record_number\": 3}, {\"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.4390042158654446, \"u_probability\": 0.9448524288198547, \"bayes_factor\": 0.46462728197012976, \"log2_bayes_factor\": -1.1058542261314457, \"comparison_vector_value\": 0, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is  2.15 times less likely to be a match\", \"column_name\": \"city\", \"value_l\": \"Lodon\", \"value_r\": \"Londonn\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 5, \"record_number\": 3}, {\"sql_condition\": \"\\\"email_l\\\" IS NULL OR \\\"email_r\\\" IS NULL\", \"label_for_charts\": \"email is NULL\", \"bayes_factor\": 1.0, \"log2_bayes_factor\": 0.0, \"comparison_vector_value\": -1, \"bayes_factor_description\": \"If comparison level is `email is null` then comparison is 1.00 times more likely to be a match\", \"column_name\": \"email\", \"value_l\": \"j.fisher4@sullivan.com\", \"value_r\": \"None\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 6, \"record_number\": 3}, {\"column_name\": \"Final score\", \"label_for_charts\": \"Final score\", \"sql_condition\": null, \"log2_bayes_factor\": -1.8127257498018765, \"bayes_factor\": 0.28465261337837, \"comparison_vector_value\": null, \"m_probability\": null, \"u_probability\": null, \"bayes_factor_description\": null, \"value_l\": \"\", \"value_r\": \"\", \"term_frequency_adjustment\": null, \"bar_sort_order\": 7, \"record_number\": 3}, {\"column_name\": \"Prior\", \"label_for_charts\": \"Starting match weight (prior)\", \"sql_condition\": null, \"log2_bayes_factor\": -13.287568102831404, \"bayes_factor\": 0.00010001000100010001, \"comparison_vector_value\": null, \"m_probability\": null, \"u_probability\": null, \"bayes_factor_description\": null, \"value_l\": \"\", \"value_r\": \"\", \"term_frequency_adjustment\": null, \"bar_sort_order\": 0, \"record_number\": 4}, {\"sql_condition\": \"\\\"first_name_l\\\" IS NULL OR \\\"first_name_r\\\" IS NULL\", \"label_for_charts\": \"first_name is NULL\", \"bayes_factor\": 1.0, \"log2_bayes_factor\": 0.0, \"comparison_vector_value\": -1, \"bayes_factor_description\": \"If comparison level is `first_name is null` then comparison is 1.00 times more likely to be a match\", \"column_name\": \"first_name\", \"value_l\": \"Michael\", \"value_r\": \"None\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 1, \"record_number\": 4}, {\"sql_condition\": \"\\\"first_name_l\\\" IS NULL OR \\\"first_name_r\\\" IS NULL\", \"label_for_charts\": \"first_name is NULL\", \"bayes_factor\": 1.0, \"log2_bayes_factor\": 0.0, \"comparison_vector_value\": -1, \"bayes_factor_description\": \"If comparison level is `first_name is null` then comparison is 1.00 times more likely to be a match\", \"column_name\": \"tf_first_name\", \"value_l\": \"\", \"value_r\": \"\", \"term_frequency_adjustment\": true, \"bar_sort_order\": 2, \"record_number\": 4}, {\"sql_condition\": \"\\\"surname_l\\\" = \\\"surname_r\\\"\", \"label_for_charts\": \"Exact match on surname\", \"m_probability\": 0.43118558905152193, \"u_probability\": 0.004889975550122249, \"bayes_factor\": 88.17745296103624, \"log2_bayes_factor\": 6.462337899652856, \"comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `exact match on surname` then comparison is 88.18 times more likely to be a match\", \"column_name\": \"surname\", \"value_l\": \"Sutherland\", \"value_r\": \"Sutherland\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 3, \"record_number\": 4}, {\"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.26762728052570295, \"u_probability\": 0.9611471471471471, \"bayes_factor\": 0.27844568994463287, \"log2_bayes_factor\": -1.8445321335278149, \"comparison_vector_value\": 0, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is  3.59 times less likely to be a match\", \"column_name\": \"dob\", \"value_l\": \"1990-07-21\", \"value_r\": \"1989-06-21\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 4, \"record_number\": 4}, {\"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.4390042158654446, \"u_probability\": 0.9448524288198547, \"bayes_factor\": 0.46462728197012976, \"log2_bayes_factor\": -1.1058542261314457, \"comparison_vector_value\": 0, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is  2.15 times less likely to be a match\", \"column_name\": \"city\", \"value_l\": \"London\", \"value_r\": \"Londno\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 5, \"record_number\": 4}, {\"sql_condition\": \"\\\"email_l\\\" = \\\"email_r\\\"\", \"label_for_charts\": \"Exact match on email\", \"m_probability\": 0.5517269116501968, \"u_probability\": 0.0021938713143283602, \"bayes_factor\": 251.4855397610705, \"log2_bayes_factor\": 7.974331638255829, \"comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `exact match on email` then comparison is 251.49 times more likely to be a match\", \"column_name\": \"email\", \"value_l\": \"michael.sutherland@hobbs.com\", \"value_r\": \"michael.sutherland@hobbs.com\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 6, \"record_number\": 4}, {\"column_name\": \"Final score\", \"label_for_charts\": \"Final score\", \"sql_condition\": null, \"log2_bayes_factor\": -1.8012849245819798, \"bayes_factor\": 0.28691893290770426, \"comparison_vector_value\": null, \"m_probability\": null, \"u_probability\": null, \"bayes_factor_description\": null, \"value_l\": \"\", \"value_r\": \"\", \"term_frequency_adjustment\": null, \"bar_sort_order\": 7, \"record_number\": 4}]}}, {\"mode\": \"vega-lite\"});\n",
              "</script>"
            ],
            "text/plain": [
              "alt.LayerChart(...)"
            ]
          },
          "execution_count": 4,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "chart"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "\n",
        "!!! info \"At a glance\"\n",
        "\n",
        "    **Useful for:** Looking at the breakdown of the match weight for a pair of records.\n",
        "\n",
        "    **API Documentation:** [waterfall_chart()](../api_docs//visualisations.md#splink.internals.linker_components.visualisations.LinkerVisualisations.waterfall_chart)\n",
        "\n",
        "    **What is needed to generate the chart?** A trained Splink model"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### What the chart shows\n",
        "\n",
        "The `waterfall_chart` shows the amount of evidence of a match that is provided by each comparison for a pair of records. Each bar represents a comparison and the corresponding amount of evidence (i.e. match weight) of a match for the pair of values displayed above the bar.\n",
        "\n",
        "??? note \"What the chart tooltip shows\"\n",
        "\n",
        "    ![](./img/waterfall_chart_tooltip.png)\n",
        "\n",
        "    The tooltip contains information based on the bar that the user is hovering over, including:\n",
        "\n",
        "    - The comparison column (or columns)\n",
        "    - The column values from the pair of records being compared\n",
        "    - The comparison level as a label, SQL statement and the corresponding comparison vector value\n",
        "    - The bayes factor (i.e. how many times more likely is a match based on this evidence)\n",
        "    - The match weight for the comparison level\n",
        "    - The cumulative match probability from the chosen comparison and all of the previous comparisons."
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "<hr>"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### How to interpret the chart\n",
        "\n",
        "The first bar (labelled \"Prior\") is the match weight if no additional knowledge of features is taken into account, and can be thought of as similar to the y-intercept in a simple regression.\n",
        "\n",
        "Each subsequent bar shows the match weight for a comparison. These bars can be positive or negative depending on whether the given comparison gives positive or negative evidence for the two records being a match.\n",
        "\n",
        "Additional bars are added for comparisons with term frequency adjustments. For example, the chart above has term frequency adjustments for `first_name` so there is an extra `tf_first_name` bar showing how the frequency of a given name impacts the amount of evidence for the two records being a match.\n",
        "\n",
        "The final bar represents total match weight for the pair of records. This match weight can also be translated into a final match probablility, and the corresponding match probability is shown on the right axis (note the logarithmic scale)."
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "<hr>"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Actions to take as a result of the chart\n",
        "\n",
        "This chart is useful for spot checking pairs of records to see if the Splink model is behaving as expected.\n",
        "\n",
        "If a pair of records look like they are incorrectly being assigned as a match/non-match, it is a sign that the Splink model is not working optimally. If this is the case, it is worth revisiting the model training step. \n",
        "\n",
        "Some common scenarios include:\n",
        "\n",
        "- If a comparison isn't capturing a specific edge case (e.g. fuzzy match), add a comparison level to capture this case and retrain the model.\n",
        "\n",
        "- If the match weight for a comparison is looking unusual, refer to the [`match_weights_chart`](./match_weights_chart.ipynb) to see the match weight in context with the rest of the comparison levels within that comparison. If it is still looking unusual, you can dig deeper with the [`parameter_estimate_comparisons_chart`](./parameter_estimate_comparisons_chart.ipynb) to see if the model training runs are consistent. If there is a lot of variation between model training sessions, this can suggest some instability in the model. In this case, try some different model training rules and/or comparison levels.\n",
        "\n",
        "- If the \"Prior\" match weight is too small or large compared to the match weight provided by the comparisons, try some different determininstic rules and recall inputs to the [`estimate_probability_two_records_match` function](../api_docs/training.md).\n",
        "\n",
        "- If you are working with a model with term frequency adjustments and want to dig deeper into the impact of term frequency on the model as a whole (i.e. not just for a single pairwise comparison), check out the [`tf_adjustment_chart`](./tf_adjustment_chart.ipynb).\n"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Worked Example"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "tags": [
          "hide_output"
        ]
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "You are using the default value for `max_pairs`, which may be too small and thus lead to inaccurate estimates for your model's u-parameters. Consider increasing to 1e8 or 1e9, which will result in more accurate estimates, but with a longer run time.\n",
            "----- Estimating u probabilities using random sampling -----\n",
            "u probability not trained for dob - Abs difference of 'transformed dob <= 1 month' (comparison vector value: 1). This usually means the comparison level was never observed in the training data.\n",
            "\n",
            "Estimated u probabilities using random sampling\n",
            "\n",
            "Your model is not yet fully trained. Missing estimates for:\n",
            "    - first_name (no m values are trained).\n",
            "    - surname (no m values are trained).\n",
            "    - dob (some u values are not trained, no m values are trained).\n",
            "    - city (no m values are trained).\n",
            "    - email (no m values are trained).\n",
            "\n",
            "----- Starting EM training session -----\n",
            "\n",
            "Estimating the m probabilities of the model by blocking on:\n",
            "(l.\"first_name\" = r.\"first_name\") AND (l.\"surname\" = r.\"surname\")\n",
            "\n",
            "Parameter estimates will be made for the following comparison(s):\n",
            "    - dob\n",
            "    - city\n",
            "    - email\n",
            "\n",
            "Parameter estimates cannot be made for the following comparison(s) since they are used in the blocking rules: \n",
            "    - first_name\n",
            "    - surname\n",
            "\n",
            "WARNING:\n",
            "Level Abs difference of 'transformed dob <= 1 month' on comparison dob not observed in dataset, unable to train m value\n",
            "\n",
            "Iteration 1: Largest change in params was -0.467 in the m_probability of dob, level `Exact match on dob`\n",
            "Iteration 2: Largest change in params was 0.142 in probability_two_random_records_match\n",
            "Iteration 3: Largest change in params was 0.0311 in probability_two_random_records_match\n",
            "Iteration 4: Largest change in params was 0.0101 in probability_two_random_records_match\n",
            "Iteration 5: Largest change in params was 0.00418 in probability_two_random_records_match\n",
            "Iteration 6: Largest change in params was 0.002 in probability_two_random_records_match\n",
            "Iteration 7: Largest change in params was 0.00105 in probability_two_random_records_match\n",
            "Iteration 8: Largest change in params was 0.000577 in probability_two_random_records_match\n",
            "Iteration 9: Largest change in params was 0.000328 in probability_two_random_records_match\n",
            "Iteration 10: Largest change in params was 0.000189 in probability_two_random_records_match\n",
            "Iteration 11: Largest change in params was 0.00011 in probability_two_random_records_match\n",
            "Iteration 12: Largest change in params was 6.46e-05 in probability_two_random_records_match\n",
            "\n",
            "EM converged after 12 iterations\n",
            "m probability not trained for dob - Abs difference of 'transformed dob <= 1 month' (comparison vector value: 1). This usually means the comparison level was never observed in the training data.\n",
            "\n",
            "Your model is not yet fully trained. Missing estimates for:\n",
            "    - first_name (no m values are trained).\n",
            "    - surname (no m values are trained).\n",
            "    - dob (some u values are not trained, some m values are not trained).\n",
            "\n",
            "----- Starting EM training session -----\n",
            "\n",
            "Estimating the m probabilities of the model by blocking on:\n",
            "l.\"dob\" = r.\"dob\"\n",
            "\n",
            "Parameter estimates will be made for the following comparison(s):\n",
            "    - first_name\n",
            "    - surname\n",
            "    - city\n",
            "    - email\n",
            "\n",
            "Parameter estimates cannot be made for the following comparison(s) since they are used in the blocking rules: \n",
            "    - dob\n",
            "\n",
            "Iteration 1: Largest change in params was 0.632 in probability_two_random_records_match\n",
            "Iteration 2: Largest change in params was 0.186 in probability_two_random_records_match\n",
            "Iteration 3: Largest change in params was 0.0812 in the m_probability of first_name, level `All other comparisons`\n",
            "Iteration 4: Largest change in params was 0.0266 in probability_two_random_records_match\n",
            "Iteration 5: Largest change in params was 0.0103 in probability_two_random_records_match\n",
            "Iteration 6: Largest change in params was 0.00452 in probability_two_random_records_match\n",
            "Iteration 7: Largest change in params was 0.00216 in probability_two_random_records_match\n",
            "Iteration 8: Largest change in params was 0.0013 in probability_two_random_records_match\n",
            "Iteration 9: Largest change in params was 0.00113 in the m_probability of first_name, level `All other comparisons`\n",
            "Iteration 10: Largest change in params was 0.000732 in probability_two_random_records_match\n",
            "Iteration 11: Largest change in params was 0.000403 in probability_two_random_records_match\n",
            "Iteration 12: Largest change in params was 0.000211 in probability_two_random_records_match\n",
            "Iteration 13: Largest change in params was 0.000109 in probability_two_random_records_match\n",
            "Iteration 14: Largest change in params was 5.62e-05 in probability_two_random_records_match\n",
            "\n",
            "EM converged after 14 iterations\n",
            "\n",
            "Your model is not yet fully trained. Missing estimates for:\n",
            "    - dob (some u values are not trained, some m values are not trained).\n",
            "\n",
            " -- WARNING --\n",
            "You have called predict(), but there are some parameter estimates which have neither been estimated or specified in your settings dictionary.  To produce predictions the following untrained trained parameters will use default values.\n",
            "Comparison: 'dob':\n",
            "    m values not fully trained\n",
            "Comparison: 'dob':\n",
            "    u values not fully trained\n",
            "The 'probability_two_random_records_match' setting has been set to the default value (0.0001). \n",
            "If this is not the desired behaviour, either: \n",
            " - assign a value for `probability_two_random_records_match` in your settings dictionary, or \n",
            " - estimate with the `linker.training.estimate_probability_two_random_records_match` function.\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "<style>\n",
              "  #altair-viz-939498fe6ab6458b932a25f0ebe677ee.vega-embed {\n",
              "    width: 100%;\n",
              "    display: flex;\n",
              "  }\n",
              "\n",
              "  #altair-viz-939498fe6ab6458b932a25f0ebe677ee.vega-embed details,\n",
              "  #altair-viz-939498fe6ab6458b932a25f0ebe677ee.vega-embed details summary {\n",
              "    position: relative;\n",
              "  }\n",
              "</style>\n",
              "<div id=\"altair-viz-939498fe6ab6458b932a25f0ebe677ee\"></div>\n",
              "<script type=\"text/javascript\">\n",
              "  var VEGA_DEBUG = (typeof VEGA_DEBUG == \"undefined\") ? {} : VEGA_DEBUG;\n",
              "  (function(spec, embedOpt){\n",
              "    let outputDiv = document.currentScript.previousElementSibling;\n",
              "    if (outputDiv.id !== \"altair-viz-939498fe6ab6458b932a25f0ebe677ee\") {\n",
              "      outputDiv = document.getElementById(\"altair-viz-939498fe6ab6458b932a25f0ebe677ee\");\n",
              "    }\n",
              "    const paths = {\n",
              "      \"vega\": \"https://cdn.jsdelivr.net/npm/vega@5?noext\",\n",
              "      \"vega-lib\": \"https://cdn.jsdelivr.net/npm/vega-lib?noext\",\n",
              "      \"vega-lite\": \"https://cdn.jsdelivr.net/npm/vega-lite@5.17.0?noext\",\n",
              "      \"vega-embed\": \"https://cdn.jsdelivr.net/npm/vega-embed@6?noext\",\n",
              "    };\n",
              "\n",
              "    function maybeLoadScript(lib, version) {\n",
              "      var key = `${lib.replace(\"-\", \"\")}_version`;\n",
              "      return (VEGA_DEBUG[key] == version) ?\n",
              "        Promise.resolve(paths[lib]) :\n",
              "        new Promise(function(resolve, reject) {\n",
              "          var s = document.createElement('script');\n",
              "          document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
              "          s.async = true;\n",
              "          s.onload = () => {\n",
              "            VEGA_DEBUG[key] = version;\n",
              "            return resolve(paths[lib]);\n",
              "          };\n",
              "          s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
              "          s.src = paths[lib];\n",
              "        });\n",
              "    }\n",
              "\n",
              "    function showError(err) {\n",
              "      outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
              "      throw err;\n",
              "    }\n",
              "\n",
              "    function displayChart(vegaEmbed) {\n",
              "      vegaEmbed(outputDiv, spec, embedOpt)\n",
              "        .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
              "    }\n",
              "\n",
              "    if(typeof define === \"function\" && define.amd) {\n",
              "      requirejs.config({paths});\n",
              "      require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
              "    } else {\n",
              "      maybeLoadScript(\"vega\", \"5\")\n",
              "        .then(() => maybeLoadScript(\"vega-lite\", \"5.17.0\"))\n",
              "        .then(() => maybeLoadScript(\"vega-embed\", \"6\"))\n",
              "        .catch(showError)\n",
              "        .then(() => displayChart(vegaEmbed));\n",
              "    }\n",
              "  })({\"config\": {\"view\": {\"continuousWidth\": 400, \"continuousHeight\": 300}}, \"layer\": [{\"layer\": [{\"mark\": \"rule\", \"encoding\": {\"color\": {\"value\": \"black\"}, \"size\": {\"value\": 0.5}, \"y\": {\"field\": \"zero\", \"type\": \"quantitative\"}}}, {\"mark\": {\"type\": \"bar\", \"width\": 60}, \"encoding\": {\"color\": {\"condition\": {\"test\": \"(datum.log2_bayes_factor < 0)\", \"value\": \"red\"}, \"value\": \"green\"}, \"opacity\": {\"condition\": {\"test\": \"datum.column_name == 'Prior match weight' || datum.column_name == 'Final score'\", \"value\": 1}, \"value\": 0.5}, \"tooltip\": [{\"field\": \"column_name\", \"title\": \"Comparison column\", \"type\": \"nominal\"}, {\"field\": \"value_l\", \"title\": \"Value (L)\", \"type\": \"nominal\"}, {\"field\": \"value_r\", \"title\": \"Value (R)\", \"type\": \"nominal\"}, {\"field\": \"label_for_charts\", \"title\": \"Label\", \"type\": \"ordinal\"}, {\"field\": \"sql_condition\", \"title\": \"SQL condition\", \"type\": \"nominal\"}, {\"field\": \"comparison_vector_value\", \"title\": \"Comparison vector value\", \"type\": \"nominal\"}, {\"field\": \"bayes_factor\", \"format\": \",.4f\", \"title\": \"Bayes factor = m/u\", \"type\": \"quantitative\"}, {\"field\": \"log2_bayes_factor\", \"format\": \",.4f\", \"title\": \"Match weight = log2(m/u)\", \"type\": \"quantitative\"}, {\"field\": \"prob\", \"format\": \".4f\", \"title\": \"Cumulative match probability\", \"type\": \"quantitative\"}, {\"field\": \"bayes_factor_description\", \"title\": \"Match weight description\", \"type\": \"nominal\"}], \"x\": {\"axis\": {\"grid\": true, \"labelAlign\": \"center\", \"labelAngle\": -20, \"labelExpr\": \"datum.value == 'Prior' || datum.value == 'Final score' ? '' : datum.value\", \"labelPadding\": 10, \"tickBand\": \"extent\", \"title\": \"Column\"}, \"field\": \"column_name\", \"sort\": {\"field\": \"bar_sort_order\", \"order\": \"ascending\"}, \"type\": \"nominal\"}, \"y\": {\"axis\": {\"grid\": false, \"orient\": \"left\", \"title\": \"Match Weight\"}, \"field\": \"previous_sum\", \"type\": \"quantitative\"}, \"y2\": {\"field\": \"sum\"}}}, {\"mark\": {\"type\": \"text\", \"fontWeight\": \"bold\"}, \"encoding\": {\"color\": {\"value\": \"white\"}, \"text\": {\"condition\": {\"test\": \"abs(datum.log2_bayes_factor) > 1\", \"field\": \"log2_bayes_factor\", \"format\": \".2f\", \"type\": \"nominal\"}, \"value\": \"\"}, \"x\": {\"axis\": {\"labelAngle\": -20, \"title\": \"Column\"}, \"field\": \"column_name\", \"sort\": {\"field\": \"bar_sort_order\", \"order\": \"ascending\"}, \"type\": \"nominal\"}, \"y\": {\"axis\": {\"orient\": \"left\"}, \"field\": \"center\", \"type\": \"quantitative\"}}}, {\"mark\": {\"type\": \"text\", \"baseline\": \"bottom\", \"dy\": -25, \"fontWeight\": \"bold\"}, \"encoding\": {\"color\": {\"value\": \"black\"}, \"text\": {\"field\": \"column_name\", \"type\": \"nominal\"}, \"x\": {\"axis\": {\"labelAngle\": -20, \"title\": \"Column\"}, \"field\": \"column_name\", \"sort\": {\"field\": \"bar_sort_order\", \"order\": \"ascending\"}, \"type\": \"nominal\"}, \"y\": {\"field\": \"sum_top\", \"type\": \"quantitative\"}}}, {\"mark\": {\"type\": \"text\", \"baseline\": \"bottom\", \"dy\": -13, \"fontSize\": 8}, \"encoding\": {\"color\": {\"value\": \"grey\"}, \"text\": {\"field\": \"value_l\", \"type\": \"nominal\"}, \"x\": {\"axis\": {\"labelAngle\": -20, \"title\": \"Column\"}, \"field\": \"column_name\", \"sort\": {\"field\": \"bar_sort_order\", \"order\": \"ascending\"}, \"type\": \"nominal\"}, \"y\": {\"field\": \"sum_top\", \"type\": \"quantitative\"}}}, {\"mark\": {\"type\": \"text\", \"baseline\": \"bottom\", \"dy\": -5, \"fontSize\": 8}, \"encoding\": {\"color\": {\"value\": \"grey\"}, \"text\": {\"field\": \"value_r\", \"type\": \"nominal\"}, \"x\": {\"axis\": {\"labelAngle\": -20, \"title\": \"Column\"}, \"field\": \"column_name\", \"sort\": {\"field\": \"bar_sort_order\", \"order\": \"ascending\"}, \"type\": \"nominal\"}, \"y\": {\"field\": \"sum_top\", \"type\": \"quantitative\"}}}]}, {\"mark\": {\"type\": \"rule\", \"color\": \"black\", \"strokeWidth\": 2, \"x2Offset\": 30, \"xOffset\": -30}, \"encoding\": {\"x\": {\"axis\": {\"labelAngle\": -20, \"title\": \"Column\"}, \"field\": \"column_name\", \"sort\": {\"field\": \"bar_sort_order\", \"order\": \"ascending\"}, \"type\": \"nominal\"}, \"x2\": {\"field\": \"lead\"}, \"y\": {\"axis\": {\"labelExpr\": \"format(1 / (1 + pow(2, -1*datum.value)), '.2r')\", \"orient\": \"right\", \"title\": \"Probability\"}, \"field\": \"sum\", \"scale\": {\"zero\": false}, \"type\": \"quantitative\"}}}], \"data\": {\"name\": \"data-61650774e7af395798e3bd6248af495f\"}, \"height\": 450, \"params\": [{\"name\": \"record_number\", \"bind\": {\"input\": \"range\", \"max\": 4, \"min\": 0, \"step\": 1}, \"value\": 0}], \"resolve\": {\"axis\": {\"y\": \"independent\"}}, \"title\": {\"text\": \"Match weights waterfall chart\", \"subtitle\": \"How each comparison contributes to the final match score\"}, \"transform\": [{\"filter\": \"(datum.record_number == record_number)\"}, {\"window\": [{\"op\": \"sum\", \"field\": \"log2_bayes_factor\", \"as\": \"sum\"}, {\"op\": \"lead\", \"field\": \"column_name\", \"as\": \"lead\"}], \"frame\": [null, 0]}, {\"calculate\": \"datum.column_name === \\\"Final score\\\" ? datum.sum - datum.log2_bayes_factor : datum.sum\", \"as\": \"sum\"}, {\"calculate\": \"datum.lead === null ? datum.column_name : datum.lead\", \"as\": \"lead\"}, {\"calculate\": \"datum.column_name === \\\"Final score\\\" || datum.column_name === \\\"Prior match weight\\\" ? 0 : datum.sum - datum.log2_bayes_factor\", \"as\": \"previous_sum\"}, {\"calculate\": \"datum.sum > datum.previous_sum ? datum.column_name : \\\"\\\"\", \"as\": \"top_label\"}, {\"calculate\": \"datum.sum < datum.previous_sum ? datum.column_name : \\\"\\\"\", \"as\": \"bottom_label\"}, {\"calculate\": \"datum.sum > datum.previous_sum ? datum.sum : datum.previous_sum\", \"as\": \"sum_top\"}, {\"calculate\": \"datum.sum < datum.previous_sum ? datum.sum : datum.previous_sum\", \"as\": \"sum_bottom\"}, {\"calculate\": \"(datum.sum + datum.previous_sum) / 2\", \"as\": \"center\"}, {\"calculate\": \"(datum.log2_bayes_factor > 0 ? \\\"+\\\" : \\\"\\\") + datum.log2_bayes_factor\", \"as\": \"text_log2_bayes_factor\"}, {\"calculate\": \"datum.sum < datum.previous_sum ? 4 : -4\", \"as\": \"dy\"}, {\"calculate\": \"datum.sum < datum.previous_sum ? \\\"top\\\" : \\\"bottom\\\"\", \"as\": \"baseline\"}, {\"calculate\": \"1. / (1 + pow(2, -1.*datum.sum))\", \"as\": \"prob\"}, {\"calculate\": \"0*datum.sum\", \"as\": \"zero\"}], \"width\": {\"step\": 75}, \"$schema\": \"https://vega.github.io/schema/vega-lite/v5.9.3.json\", \"datasets\": {\"data-61650774e7af395798e3bd6248af495f\": [{\"column_name\": \"Prior\", \"label_for_charts\": \"Starting match weight (prior)\", \"sql_condition\": null, \"log2_bayes_factor\": -13.287568102831404, \"bayes_factor\": 0.00010001000100010001, \"comparison_vector_value\": null, \"m_probability\": null, \"u_probability\": null, \"bayes_factor_description\": null, \"value_l\": \"\", \"value_r\": \"\", \"term_frequency_adjustment\": null, \"bar_sort_order\": 0, \"record_number\": 0}, {\"sql_condition\": \"\\\"first_name_l\\\" IS NULL OR \\\"first_name_r\\\" IS NULL\", \"label_for_charts\": \"first_name is NULL\", \"bayes_factor\": 1.0, \"log2_bayes_factor\": 0.0, \"comparison_vector_value\": -1, \"bayes_factor_description\": \"If comparison level is `first_name is null` then comparison is 1.00 times more likely to be a match\", \"column_name\": \"first_name\", \"value_l\": \"None\", \"value_r\": \"Lola\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 1, \"record_number\": 0}, {\"sql_condition\": \"\\\"first_name_l\\\" IS NULL OR \\\"first_name_r\\\" IS NULL\", \"label_for_charts\": \"first_name is NULL\", \"bayes_factor\": 1.0, \"log2_bayes_factor\": 0.0, \"comparison_vector_value\": -1, \"bayes_factor_description\": \"If comparison level is `first_name is null` then comparison is 1.00 times more likely to be a match\", \"column_name\": \"tf_first_name\", \"value_l\": \"\", \"value_r\": \"\", \"term_frequency_adjustment\": true, \"bar_sort_order\": 2, \"record_number\": 0}, {\"sql_condition\": \"\\\"surname_l\\\" = \\\"surname_r\\\"\", \"label_for_charts\": \"Exact match on surname\", \"m_probability\": 0.43118558905152193, \"u_probability\": 0.004889975550122249, \"bayes_factor\": 88.17745296103624, \"log2_bayes_factor\": 6.462337899652856, \"comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `exact match on surname` then comparison is 88.18 times more likely to be a match\", \"column_name\": \"surname\", \"value_l\": \"Taylor\", \"value_r\": \"Taylor\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 3, \"record_number\": 0}, {\"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.26762728052570295, \"u_probability\": 0.9611471471471471, \"bayes_factor\": 0.27844568994463287, \"log2_bayes_factor\": -1.8445321335278149, \"comparison_vector_value\": 0, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is  3.59 times less likely to be a match\", \"column_name\": \"dob\", \"value_l\": \"2016-11-20\", \"value_r\": \"2017-11-21\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 4, \"record_number\": 0}, {\"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.4390042158654446, \"u_probability\": 0.9448524288198547, \"bayes_factor\": 0.46462728197012976, \"log2_bayes_factor\": -1.1058542261314457, \"comparison_vector_value\": 0, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is  2.15 times less likely to be a match\", \"column_name\": \"city\", \"value_l\": \"Abereden\", \"value_r\": \"Aberdeen\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 5, \"record_number\": 0}, {\"sql_condition\": \"jaro_winkler_similarity(\\\"email_l\\\", \\\"email_r\\\") >= 0.88\", \"label_for_charts\": \"Jaro-Winkler distance of email >= 0.88\", \"m_probability\": 0.21355998957004996, \"u_probability\": 0.0009135769109519858, \"bayes_factor\": 233.76246379465897, \"log2_bayes_factor\": 7.868899478733789, \"comparison_vector_value\": 1, \"bayes_factor_description\": \"If comparison level is `jaro-winkler distance of email >= 0.88` then comparison is 233.76 times more likely to be a match\", \"column_name\": \"email\", \"value_l\": \"glolat86@bishop-giles.com\", \"value_r\": \"lolat86@bishop-giles.om\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 6, \"record_number\": 0}, {\"column_name\": \"Final score\", \"label_for_charts\": \"Final score\", \"sql_condition\": null, \"log2_bayes_factor\": -1.9067170841040197, \"bayes_factor\": 0.2666987403313988, \"comparison_vector_value\": null, \"m_probability\": null, \"u_probability\": null, \"bayes_factor_description\": null, \"value_l\": \"\", \"value_r\": \"\", \"term_frequency_adjustment\": null, \"bar_sort_order\": 7, \"record_number\": 0}, {\"column_name\": \"Prior\", \"label_for_charts\": \"Starting match weight (prior)\", \"sql_condition\": null, \"log2_bayes_factor\": -13.287568102831404, \"bayes_factor\": 0.00010001000100010001, \"comparison_vector_value\": null, \"m_probability\": null, \"u_probability\": null, \"bayes_factor_description\": null, \"value_l\": \"\", \"value_r\": \"\", \"term_frequency_adjustment\": null, \"bar_sort_order\": 0, \"record_number\": 1}, {\"sql_condition\": \"\\\"first_name_l\\\" = \\\"first_name_r\\\"\", \"label_for_charts\": \"Exact match on first_name\", \"m_probability\": 0.4874498174690064, \"u_probability\": 0.0057935713975033705, \"bayes_factor\": 84.13632697770215, \"log2_bayes_factor\": 6.394656932643231, \"comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `exact match on first_name` then comparison is 84.14 times more likely to be a match\", \"column_name\": \"first_name\", \"value_l\": \"Eleanor\", \"value_r\": \"Eleanor\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 1, \"record_number\": 1}, {\"sql_condition\": \"\\\"first_name_l\\\" = \\\"first_name_r\\\"\", \"label_for_charts\": \"Term freq adjustment on first_name with weight {cl.tf_adjustment_weight}\", \"m_probability\": null, \"u_probability\": null, \"bayes_factor\": 1.2036144578313253, \"log2_bayes_factor\": 0.2673733415581312, \"comparison_vector_value\": 3, \"bayes_factor_description\": \"Term frequency adjustment on first_name makes comparison 1.20 times more likely to be a match\", \"column_name\": \"tf_first_name\", \"value_l\": \"Eleanor\", \"value_r\": \"Eleanor\", \"term_frequency_adjustment\": true, \"bar_sort_order\": 2, \"record_number\": 1}, {\"sql_condition\": \"\\\"surname_l\\\" IS NULL OR \\\"surname_r\\\" IS NULL\", \"label_for_charts\": \"surname is NULL\", \"bayes_factor\": 1.0, \"log2_bayes_factor\": 0.0, \"comparison_vector_value\": -1, \"bayes_factor_description\": \"If comparison level is `surname is null` then comparison is 1.00 times more likely to be a match\", \"column_name\": \"surname\", \"value_l\": \"None\", \"value_r\": \"Richards\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 3, \"record_number\": 1}, {\"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.26762728052570295, \"u_probability\": 0.9611471471471471, \"bayes_factor\": 0.27844568994463287, \"log2_bayes_factor\": -1.8445321335278149, \"comparison_vector_value\": 0, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is  3.59 times less likely to be a match\", \"column_name\": \"dob\", \"value_l\": \"2024-07-07\", \"value_r\": \"2014-07-10\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 4, \"record_number\": 1}, {\"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.4390042158654446, \"u_probability\": 0.9448524288198547, \"bayes_factor\": 0.46462728197012976, \"log2_bayes_factor\": -1.1058542261314457, \"comparison_vector_value\": 0, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is  2.15 times less likely to be a match\", \"column_name\": \"city\", \"value_l\": \"Manchester\", \"value_r\": \"Mancester\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 5, \"record_number\": 1}, {\"sql_condition\": \"NULLIF(regexp_extract(\\\"email_l\\\", '^[^@]+', 0), '') = NULLIF(regexp_extract(\\\"email_r\\\", '^[^@]+', 0), '')\", \"label_for_charts\": \"Exact match on transformed email\", \"m_probability\": 0.22027963026744735, \"u_probability\": 0.0010390328952024346, \"bayes_factor\": 212.00448155640953, \"log2_bayes_factor\": 7.727950951972963, \"comparison_vector_value\": 2, \"bayes_factor_description\": \"If comparison level is `exact match on transformed email` then comparison is 212.00 times more likely to be a match\", \"column_name\": \"email\", \"value_l\": \"e.richards16@finley.ifo\", \"value_r\": \"e.richards16@finley.info\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 6, \"record_number\": 1}, {\"column_name\": \"Final score\", \"label_for_charts\": \"Final score\", \"sql_condition\": null, \"log2_bayes_factor\": -1.8479732363163406, \"bayes_factor\": 0.2777823353038195, \"comparison_vector_value\": null, \"m_probability\": null, \"u_probability\": null, \"bayes_factor_description\": null, \"value_l\": \"\", \"value_r\": \"\", \"term_frequency_adjustment\": null, \"bar_sort_order\": 7, \"record_number\": 1}, {\"column_name\": \"Prior\", \"label_for_charts\": \"Starting match weight (prior)\", \"sql_condition\": null, \"log2_bayes_factor\": -13.287568102831404, \"bayes_factor\": 0.00010001000100010001, \"comparison_vector_value\": null, \"m_probability\": null, \"u_probability\": null, \"bayes_factor_description\": null, \"value_l\": \"\", \"value_r\": \"\", \"term_frequency_adjustment\": null, \"bar_sort_order\": 0, \"record_number\": 2}, {\"sql_condition\": \"\\\"first_name_l\\\" = \\\"first_name_r\\\"\", \"label_for_charts\": \"Exact match on first_name\", \"m_probability\": 0.4874498174690064, \"u_probability\": 0.0057935713975033705, \"bayes_factor\": 84.13632697770215, \"log2_bayes_factor\": 6.394656932643231, \"comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `exact match on first_name` then comparison is 84.14 times more likely to be a match\", \"column_name\": \"first_name\", \"value_l\": \"Darcy\", \"value_r\": \"Darcy\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 1, \"record_number\": 2}, {\"sql_condition\": \"\\\"first_name_l\\\" = \\\"first_name_r\\\"\", \"label_for_charts\": \"Term freq adjustment on first_name with weight {cl.tf_adjustment_weight}\", \"m_probability\": null, \"u_probability\": null, \"bayes_factor\": 0.8024096385542168, \"log2_bayes_factor\": -0.31758915916302516, \"comparison_vector_value\": 3, \"bayes_factor_description\": \"Term frequency adjustment on first_name makes comparison  1.25 times less likely to be a match\", \"column_name\": \"tf_first_name\", \"value_l\": \"Darcy\", \"value_r\": \"Darcy\", \"term_frequency_adjustment\": true, \"bar_sort_order\": 2, \"record_number\": 2}, {\"sql_condition\": \"\\\"surname_l\\\" IS NULL OR \\\"surname_r\\\" IS NULL\", \"label_for_charts\": \"surname is NULL\", \"bayes_factor\": 1.0, \"log2_bayes_factor\": 0.0, \"comparison_vector_value\": -1, \"bayes_factor_description\": \"If comparison level is `surname is null` then comparison is 1.00 times more likely to be a match\", \"column_name\": \"surname\", \"value_l\": \"None\", \"value_r\": \"Rhodes\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 3, \"record_number\": 2}, {\"sql_condition\": \"damerau_levenshtein(\\\"dob_l\\\", \\\"dob_r\\\") <= 1\", \"label_for_charts\": \"Damerau-Levenshtein distance of dob <= 1\", \"m_probability\": 0.14875392956459, \"u_probability\": 0.0016436436436436436, \"bayes_factor\": 90.50254301767687, \"log2_bayes_factor\": 6.499886425729021, \"comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `damerau-levenshtein distance of dob <= 1` then comparison is 90.50 times more likely to be a match\", \"column_name\": \"dob\", \"value_l\": \"1979-01-16\", \"value_r\": \"1979-01-14\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 4, \"record_number\": 2}, {\"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.4390042158654446, \"u_probability\": 0.9448524288198547, \"bayes_factor\": 0.46462728197012976, \"log2_bayes_factor\": -1.1058542261314457, \"comparison_vector_value\": 0, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is  2.15 times less likely to be a match\", \"column_name\": \"city\", \"value_l\": \"nBirminham\", \"value_r\": \"Birmngham\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 5, \"record_number\": 2}, {\"sql_condition\": \"\\\"email_l\\\" IS NULL OR \\\"email_r\\\" IS NULL\", \"label_for_charts\": \"email is NULL\", \"bayes_factor\": 1.0, \"log2_bayes_factor\": 0.0, \"comparison_vector_value\": -1, \"bayes_factor_description\": \"If comparison level is `email is null` then comparison is 1.00 times more likely to be a match\", \"column_name\": \"email\", \"value_l\": \"drhodes16@johnson-robinson.com\", \"value_r\": \"None\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 6, \"record_number\": 2}, {\"column_name\": \"Final score\", \"label_for_charts\": \"Final score\", \"sql_condition\": null, \"log2_bayes_factor\": -1.816468129753623, \"bayes_factor\": 0.2839151756519555, \"comparison_vector_value\": null, \"m_probability\": null, \"u_probability\": null, \"bayes_factor_description\": null, \"value_l\": \"\", \"value_r\": \"\", \"term_frequency_adjustment\": null, \"bar_sort_order\": 7, \"record_number\": 2}, {\"column_name\": \"Prior\", \"label_for_charts\": \"Starting match weight (prior)\", \"sql_condition\": null, \"log2_bayes_factor\": -13.287568102831404, \"bayes_factor\": 0.00010001000100010001, \"comparison_vector_value\": null, \"m_probability\": null, \"u_probability\": null, \"bayes_factor_description\": null, \"value_l\": \"\", \"value_r\": \"\", \"term_frequency_adjustment\": null, \"bar_sort_order\": 0, \"record_number\": 3}, {\"sql_condition\": \"\\\"first_name_l\\\" = \\\"first_name_r\\\"\", \"label_for_charts\": \"Exact match on first_name\", \"m_probability\": 0.4874498174690064, \"u_probability\": 0.0057935713975033705, \"bayes_factor\": 84.13632697770215, \"log2_bayes_factor\": 6.394656932643231, \"comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `exact match on first_name` then comparison is 84.14 times more likely to be a match\", \"column_name\": \"first_name\", \"value_l\": \"Jackson\", \"value_r\": \"Jackson\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 1, \"record_number\": 3}, {\"sql_condition\": \"\\\"first_name_l\\\" = \\\"first_name_r\\\"\", \"label_for_charts\": \"Term freq adjustment on first_name with weight {cl.tf_adjustment_weight}\", \"m_probability\": null, \"u_probability\": null, \"bayes_factor\": 0.6877796901893287, \"log2_bayes_factor\": -0.539981580499473, \"comparison_vector_value\": 3, \"bayes_factor_description\": \"Term frequency adjustment on first_name makes comparison  1.45 times less likely to be a match\", \"column_name\": \"tf_first_name\", \"value_l\": \"Jackson\", \"value_r\": \"Jackson\", \"term_frequency_adjustment\": true, \"bar_sort_order\": 2, \"record_number\": 3}, {\"sql_condition\": \"jaro_winkler_similarity(\\\"surname_l\\\", \\\"surname_r\\\") >= 0.8\", \"label_for_charts\": \"Jaro-Winkler distance of surname >= 0.8\", \"m_probability\": 0.07560521862782084, \"u_probability\": 0.0039048156407569612, \"bayes_factor\": 19.362045633776585, \"log2_bayes_factor\": 4.275159478791109, \"comparison_vector_value\": 1, \"bayes_factor_description\": \"If comparison level is `jaro-winkler distance of surname >= 0.8` then comparison is 19.36 times more likely to be a match\", \"column_name\": \"surname\", \"value_l\": \"Fisreh\", \"value_r\": \"Fishier\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 3, \"record_number\": 3}, {\"sql_condition\": \"ABS(EPOCH(try_strptime(\\\"dob_l\\\", '%Y-%m-%d')) - EPOCH(try_strptime(\\\"dob_r\\\", '%Y-%m-%d'))) <= 31557600.0\", \"label_for_charts\": \"Abs difference of 'transformed dob <= 1 year'\", \"m_probability\": 0.19388291091910234, \"u_probability\": 0.03546146146146146, \"bayes_factor\": 5.46742584565526, \"log2_bayes_factor\": 2.4508617482261057, \"comparison_vector_value\": 2, \"bayes_factor_description\": \"If comparison level is `abs difference of 'transformed dob <= 1 year'` then comparison is 5.47 times more likely to be a match\", \"column_name\": \"dob\", \"value_l\": \"2004-12-26\", \"value_r\": \"2003-12-28\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 4, \"record_number\": 3}, {\"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.4390042158654446, \"u_probability\": 0.9448524288198547, \"bayes_factor\": 0.46462728197012976, \"log2_bayes_factor\": -1.1058542261314457, \"comparison_vector_value\": 0, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is  2.15 times less likely to be a match\", \"column_name\": \"city\", \"value_l\": \"Lodon\", \"value_r\": \"Londonn\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 5, \"record_number\": 3}, {\"sql_condition\": \"\\\"email_l\\\" IS NULL OR \\\"email_r\\\" IS NULL\", \"label_for_charts\": \"email is NULL\", \"bayes_factor\": 1.0, \"log2_bayes_factor\": 0.0, \"comparison_vector_value\": -1, \"bayes_factor_description\": \"If comparison level is `email is null` then comparison is 1.00 times more likely to be a match\", \"column_name\": \"email\", \"value_l\": \"j.fisher4@sullivan.com\", \"value_r\": \"None\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 6, \"record_number\": 3}, {\"column_name\": \"Final score\", \"label_for_charts\": \"Final score\", \"sql_condition\": null, \"log2_bayes_factor\": -1.8127257498018765, \"bayes_factor\": 0.28465261337837, \"comparison_vector_value\": null, \"m_probability\": null, \"u_probability\": null, \"bayes_factor_description\": null, \"value_l\": \"\", \"value_r\": \"\", \"term_frequency_adjustment\": null, \"bar_sort_order\": 7, \"record_number\": 3}, {\"column_name\": \"Prior\", \"label_for_charts\": \"Starting match weight (prior)\", \"sql_condition\": null, \"log2_bayes_factor\": -13.287568102831404, \"bayes_factor\": 0.00010001000100010001, \"comparison_vector_value\": null, \"m_probability\": null, \"u_probability\": null, \"bayes_factor_description\": null, \"value_l\": \"\", \"value_r\": \"\", \"term_frequency_adjustment\": null, \"bar_sort_order\": 0, \"record_number\": 4}, {\"sql_condition\": \"\\\"first_name_l\\\" IS NULL OR \\\"first_name_r\\\" IS NULL\", \"label_for_charts\": \"first_name is NULL\", \"bayes_factor\": 1.0, \"log2_bayes_factor\": 0.0, \"comparison_vector_value\": -1, \"bayes_factor_description\": \"If comparison level is `first_name is null` then comparison is 1.00 times more likely to be a match\", \"column_name\": \"first_name\", \"value_l\": \"Michael\", \"value_r\": \"None\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 1, \"record_number\": 4}, {\"sql_condition\": \"\\\"first_name_l\\\" IS NULL OR \\\"first_name_r\\\" IS NULL\", \"label_for_charts\": \"first_name is NULL\", \"bayes_factor\": 1.0, \"log2_bayes_factor\": 0.0, \"comparison_vector_value\": -1, \"bayes_factor_description\": \"If comparison level is `first_name is null` then comparison is 1.00 times more likely to be a match\", \"column_name\": \"tf_first_name\", \"value_l\": \"\", \"value_r\": \"\", \"term_frequency_adjustment\": true, \"bar_sort_order\": 2, \"record_number\": 4}, {\"sql_condition\": \"\\\"surname_l\\\" = \\\"surname_r\\\"\", \"label_for_charts\": \"Exact match on surname\", \"m_probability\": 0.43118558905152193, \"u_probability\": 0.004889975550122249, \"bayes_factor\": 88.17745296103624, \"log2_bayes_factor\": 6.462337899652856, \"comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `exact match on surname` then comparison is 88.18 times more likely to be a match\", \"column_name\": \"surname\", \"value_l\": \"Sutherland\", \"value_r\": \"Sutherland\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 3, \"record_number\": 4}, {\"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.26762728052570295, \"u_probability\": 0.9611471471471471, \"bayes_factor\": 0.27844568994463287, \"log2_bayes_factor\": -1.8445321335278149, \"comparison_vector_value\": 0, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is  3.59 times less likely to be a match\", \"column_name\": \"dob\", \"value_l\": \"1990-07-21\", \"value_r\": \"1989-06-21\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 4, \"record_number\": 4}, {\"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.4390042158654446, \"u_probability\": 0.9448524288198547, \"bayes_factor\": 0.46462728197012976, \"log2_bayes_factor\": -1.1058542261314457, \"comparison_vector_value\": 0, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is  2.15 times less likely to be a match\", \"column_name\": \"city\", \"value_l\": \"London\", \"value_r\": \"Londno\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 5, \"record_number\": 4}, {\"sql_condition\": \"\\\"email_l\\\" = \\\"email_r\\\"\", \"label_for_charts\": \"Exact match on email\", \"m_probability\": 0.5517269116501968, \"u_probability\": 0.0021938713143283602, \"bayes_factor\": 251.4855397610705, \"log2_bayes_factor\": 7.974331638255829, \"comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `exact match on email` then comparison is 251.49 times more likely to be a match\", \"column_name\": \"email\", \"value_l\": \"michael.sutherland@hobbs.com\", \"value_r\": \"michael.sutherland@hobbs.com\", \"term_frequency_adjustment\": false, \"bar_sort_order\": 6, \"record_number\": 4}, {\"column_name\": \"Final score\", \"label_for_charts\": \"Final score\", \"sql_condition\": null, \"log2_bayes_factor\": -1.8012849245819798, \"bayes_factor\": 0.28691893290770426, \"comparison_vector_value\": null, \"m_probability\": null, \"u_probability\": null, \"bayes_factor_description\": null, \"value_l\": \"\", \"value_r\": \"\", \"term_frequency_adjustment\": null, \"bar_sort_order\": 7, \"record_number\": 4}]}}, {\"mode\": \"vega-lite\"});\n",
              "</script>"
            ],
            "text/plain": [
              "alt.LayerChart(...)"
            ]
          },
          "execution_count": 3,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import splink.comparison_library as cl\n",
        "from splink import DuckDBAPI, Linker, SettingsCreator, block_on, splink_datasets\n",
        "\n",
        "df = splink_datasets.fake_1000\n",
        "\n",
        "settings = SettingsCreator(\n",
        "    link_type=\"dedupe_only\",\n",
        "    comparisons=[\n",
        "        cl.NameComparison(\"first_name\").configure(term_frequency_adjustments=True),\n",
        "        cl.NameComparison(\"surname\"),\n",
        "        cl.DateOfBirthComparison(\n",
        "            \"dob\",\n",
        "            input_is_string=True,\n",
        "            datetime_metrics=[\"year\", \"month\"],\n",
        "            datetime_thresholds=[1, 1],\n",
        "        ),\n",
        "        cl.ExactMatch(\"city\"),\n",
        "        cl.EmailComparison(\"email\", include_username_fuzzy_level=False),\n",
        "    ],\n",
        "    blocking_rules_to_generate_predictions=[\n",
        "        block_on(\"first_name\"),\n",
        "        block_on(\"surname\"),\n",
        "    ],\n",
        "    retain_intermediate_calculation_columns=True,\n",
        "    retain_matching_columns=True,\n",
        ")\n",
        "\n",
        "linker = Linker(df, settings, DuckDBAPI())\n",
        "linker.training.estimate_u_using_random_sampling(max_pairs=1e6)\n",
        "\n",
        "blocking_rule_for_training = block_on(\"first_name\", \"surname\")\n",
        "linker.training.estimate_parameters_using_expectation_maximisation(\n",
        "    blocking_rule_for_training\n",
        ")\n",
        "\n",
        "blocking_rule_for_training = block_on(\"dob\")\n",
        "linker.training.estimate_parameters_using_expectation_maximisation(\n",
        "    blocking_rule_for_training\n",
        ")\n",
        "\n",
        "df_predictions = linker.inference.predict(threshold_match_probability=0.2)\n",
        "records_to_view = df_predictions.as_record_dict(limit=5)\n",
        "\n",
        "chart = linker.visualisations.waterfall_chart(records_to_view, filter_nulls=False)\n",
        "chart"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "base",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.8"
    },
    "orig_nbformat": 4
  },
  "nbformat": 4,
  "nbformat_minor": 2
}
